/**
 * Copyright (c) 2010 Andrey Somov
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.yaml.scalayaml.scanner

import java.io.Reader
import scala.collection.mutable.{Map, ArrayStack, LinkedHashMap,  ArrayBuffer}

import org.yaml.scalayaml.tokens._
import org.yaml.scalayaml.error.{Mark, MarkedYAMLException, YAMLException}
import org.yaml.scalayaml.reader.StreamReader

case class SimpleKey(tokenNumber:Int, required:Boolean, index:Int, line:Int, column:Int, mark:Mark)
case class ChompTail (empty: Boolean, value: Boolean = true) {
    def chompTailIsNotFalse = {
        empty || value
    }
    
    def chompTailIsTrue = {
        !empty && value
    }
}

case class ScannerException(context: String, contextMark: Mark, problem: String, problemMark: Mark, note: String = "")
    extends MarkedYAMLException(context, contextMark, problem, problemMark, note, null)

/**
 * This trait represents an input stream of {@link Token Tokens}.
 * <p>
 * The parser and the scanner form together the 'Parse' step in the loading
 * process (see chapter 3.1 of the <a href="http://yaml.org/spec/1.1/">YAML
 * Specification</a>).
 * </p>
 * 
 * @see org.yaml.scalayaml.tokens.Token
 * <pre>
 * Scanner produces tokens of the following types:
 * STREAM-START
 * STREAM-END
 * DIRECTIVE(name, value)
 * DOCUMENT-START
 * DOCUMENT-END
 * BLOCK-SEQUENCE-START
 * BLOCK-MAPPING-START
 * BLOCK-END
 * FLOW-SEQUENCE-START
 * FLOW-MAPPING-START
 * FLOW-SEQUENCE-END
 * FLOW-MAPPING-END
 * BLOCK-ENTRY
 * FLOW-ENTRY
 * KEY
 * VALUE
 * ALIAS(value)
 * ANCHOR(value)
 * TAG(value)
 * SCALAR(value, plain, style)
 * Read comments in the Scanner code for more details.
 * </pre>
 * 
 * @see <a href="http://pyyaml.org/wiki/PyYAML">PyYAML</a> for more information
 */
trait Scanner extends StreamReader {
    // Had we reached the end of the stream?
    private var done = false

    // The number of unclosed '{' and '['. `flow_level == 0` means block
    // context.
    private var flowLevel = 0

    // List of processed tokens that are not yet emitted.
    private var tokens: ArrayBuffer[Token] = new  ArrayBuffer[Token](100)

    // Number of tokens that were emitted through the `get_token` method.
    private var tokensTaken = 0

    // The current indentation level.
    private var indent = -1

    // Past indentation levels.
    private var indents:  ArrayStack[Int] = new  ArrayStack[Int]

    // Variables related to simple keys treatment. See PyYAML.

    /**
     * <pre>
     * A simple key is a key that is not denoted by the '?' indicator.
     * Example of simple keys:
     *   ---
     *   block simple key: value
     *   ? not a simple key:
     *   : { flow simple key: value }
     * We emit the KEY token before all keys, so when we find a potential
     * simple key, we try to locate the corresponding ':' indicator.
     * Simple keys should be limited to a single line and 1024 characters.
     * 
     * Can a simple key start at the current position? A simple key may
     * start:
     * - at the beginning of the line, not counting indentation spaces
     *       (in block context),
     * - after '{', '[', ',' (in the flow context),
     * - after '?', ':', '-' (in the block context).
     * In the block context, this flag also signifies if a block collection
     * may start at the current position.
     * </pre>
     */
    var allowSimpleKey = true

    /*
     * Keep track of possible simple keys. This is a dictionary. The key is
     * `flow_level`; there can be no more that one possible simple key for each
     * level. The value is a SimpleKey record: (token_number, required, index,
     * line, column, mark) A simple key may start with ALIAS, ANCHOR, TAG,
     * SCALAR(flow), '[', or '{' tokens.
     */
    // the order in possibleSimpleKeys is kept for nextPossibleSimpleKey()
    private var possibleSimpleKeys: Map[Int, SimpleKey] = new LinkedHashMap[Int, SimpleKey]()
    
    fetchStreamStart() // Add the STREAM-START token.
    
    // end of constructor
    
    /**
     * Check if the next token is one of the given types.
     * 
     * @param choices
     *            List of token types.
     * @return <code>true</code> if the next token can be assigned to a variable
     *         of at least one of the given types. Returns <code>false</code> if
     *         no more tokens are available.
     * @throws ScannerException
     *             Thrown in case of malformed input.
     */
    def checkToken(choices: Token.ID*): Boolean = {
        while (needMoreTokens) {
            fetchMoreTokens
        }
        if (!tokens.isEmpty) {
            if (choices.size == 0) {
                return true
            }
            val first = tokens(0)
            choices.exists(first is _)
        }
        false
    }
    
    /**
     * Return the next token, but do not delete it from the stream.
     * 
     * @return The token that will be returned on the next call to
     *         {@link #getToken}
     * @throws ScannerException
     *             Thrown in case of malformed input.
     */
    def peekToken(): Token = {
        while (needMoreTokens) {
            fetchMoreTokens
        }
        tokens(0) 
    }
    
    /**
     * Returns the next token.
     * <p>
     * The token will be removed from the stream.
     * </p>
     * 
     * @throws ScannerException
     *             Thrown in case of malformed input.
     */
    def getToken(): Token = {
        if (!tokens.isEmpty) {
            tokensTaken += 1
            tokens remove 0
        } else {
            null
        }
    }
    
    // Private methods.
    
    private def needMoreTokens(): Boolean = {
        if (done) return false
        if (tokens.isEmpty) return true
        // The current token may be a potential simple key, so we
        // need to look further.
        stalePossibleSimpleKeys()
        return nextPossibleSimpleKey == tokensTaken
    }
    
    private def fetchMoreTokens() {
        // Eat whitespaces and comments until we reach the next token.
        scanToNextToken
        // Remove obsolete possible simple keys.
        stalePossibleSimpleKeys
        // Compare the current indentation and column. It may add some tokens
        // and decrease the current indentation level.
        unwindIndent(column)
        // Peek the next character.
        val ch = peek()
        ch match {
        case '\0' => 
            // Is it the end of stream?
            fetchStreamEnd
            return
        case '%' =>
            // Is it a directive?
            if (checkDirective) {
                fetchDirective
                return
            }
        case '-' =>
            // Is it the document start?
            if (checkDocumentStart) {
                fetchDocumentStart
                return
                // Is it the block entry indicator?
            } else if (checkBlockEntry) {
                fetchBlockEntry
                return
            }
        case '.' =>
            // Is it the document end?
            if (checkDocumentEnd) {
                fetchDocumentEnd
                return
            }
        // TODO support for BOM within a stream. (not implemented in PyYAML)
        case '[' =>
            // Is it the flow sequence start indicator?
            fetchFlowSequenceStart
            return
        case '{' =>
            // Is it the flow mapping start indicator?
            fetchFlowMappingStart
            return
        case ']' =>
            // Is it the flow sequence end indicator?
            fetchFlowSequenceEnd
            return
        case '}' =>
            // Is it the flow mapping end indicator?
            fetchFlowMappingEnd
            return
        case ',' =>
            // Is it the flow entry indicator?
            fetchFlowEntry
            return
            // see block entry indicator above
        case '?' =>
            // Is it the key indicator?
            if (checkKey) {
                fetchKey
                return
            }
        case ':' =>
            // Is it the value indicator?
            if (checkValue) {
                fetchValue
                return
            }
        case '*' =>
            // Is it an alias?
            fetchAlias
            return
        case '&' =>
            // Is it an anchor?
            fetchAnchor
            return
        case '!' =>
            // Is it a tag?
            fetchTag
            return
        case '|' =>
            // Is it a literal scalar?
            if (flowLevel == 0) {
                fetchLiteral
                return
            }
        case '>' =>
            // Is it a folded scalar?
            if (flowLevel == 0) {
                fetchFolded
                return
            }
        case '\'' =>
            // Is it a single quoted scalar?
            fetchSingle
            return
        case '"' =>
            // Is it a double quoted scalar?
            fetchDouble
            return
        case _ =>
            // go further
        }
        // It must be a plain scalar then.
        if (checkPlain) {
            fetchPlain
            return
        }
        // No? It's an error. Let's produce a nice error message.
        var chRepresentation = String valueOf ch
        //if it is a special character add '\' to make is visible
        val escape = Constant.ESCAPE_REPLACEMENTS.find(p => p._2 == chRepresentation)
        if (escape != None) {
            chRepresentation = "\\" + escape.get._1 // '\u0009' -> '\' + 't'
        }
        throw new ScannerException("while scanning for the next token", null, "found character "
                + ch + "'" + chRepresentation + "' that cannot start any token", getMark)

    }
    
    /**
     * Return the number of the nearest possible simple key.
     */
    private def nextPossibleSimpleKey = {
        /*
         * the implementation is not as in PyYAML. Because
         * this.possibleSimpleKeys is ordered we can simply take the first key
         */
        if (!possibleSimpleKeys.isEmpty) {
            val simplKey = possibleSimpleKeys.head._2
            simplKey.tokenNumber
        } else {
            -1
        }
    }
    
    /**
     * <pre>
     * Remove entries that are no longer possible simple keys. According to
     * the YAML specification, simple keys
     * - should be limited to a single line,
     * - should be no longer than 1024 characters.
     * Disabling this procedure will allow simple keys of any length and
     * height (may cause problems if indentation is broken though).
     * </pre>
     */
    private def stalePossibleSimpleKeys() {
        possibleSimpleKeys = possibleSimpleKeys.filter(p => check(p._2))
        def check(key: SimpleKey) = {
            if (key.line != line || index - key.index > 1024) {
                if (key.required) throw new ScannerException("while scanning a simple key", key.mark,
                            "could not found expected ':'", getMark)
                false
            } else {
                true
            }
        }
    }
    
    /**
     * The next token may start a simple key. We check if it's possible and save
     * its position. This function is called for ALIAS, ANCHOR, TAG,
     * SCALAR(flow), '[', and '{'.
     */
    private def savePossibleSimpleKey() {
        // The next token may start a simple key. We check if it's possible
        // and save its position. This function is called for
        // ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.

        // Check if a simple key is required at the current position.
        val required = (flowLevel == 0) && (indent == column)

        if (allowSimpleKey || !required) {
            // A simple key is required only if it is the first token in the
            // current
            // line. Therefore it is always allowed.
        } else {
            throw new YAMLException(
                    "A simple key is required only if it is the first token in the current line")
        }

        // The next token might be a simple key. Let's save it's number and
        // position.
        if (allowSimpleKey) {
            removePossibleSimpleKey()
            val tokenNumber = tokensTaken + tokens.size
            val key = new SimpleKey(tokenNumber, required, index, line, column, getMark)
            possibleSimpleKeys(flowLevel) = key
        }
    }
    
    /**
     * Remove the saved possible key position at the current flow level.
     */
    private def removePossibleSimpleKey() {
        val key = possibleSimpleKeys.get(flowLevel)
        if (key != None && key.get.required) {
            throw new ScannerException("while scanning a simple key", key.get.mark,
                    "could not found expected ':'", getMark)
        }
        possibleSimpleKeys.remove(flowLevel)
    }
    
    // Indentation functions.
    
    /**
     * <pre>
     * In flow context, tokens should respect indentation.
     * Actually the condition should be `self.indent &gt;= column` according to
     * the spec. But this condition will prohibit intuitively correct
     * constructions such as
     * key : {
     * }
     * </pre>
     */
    private def unwindIndent (col: Int) {
        // In the flow context, indentation is ignored. We make the scanner less
        // restrictive then specification requires.
        if (flowLevel != 0) {
            return
        }

        // In block context, we may need to issue the BLOCK-END tokens.
        while (indent > col) {
            val mark = getMark
            indent = indents.pop
            tokens += new BlockEndToken(mark, mark)
        }
    }
    
    /**
     * Check if we need to increase indentation.
     */
    private def addIndent(column: Int) = {
        if (indent < column) {
            indents.push(indent)
            indent = column
            true
        } else {
            false
        }
    }
    
    // Fetchers.
    
    /**
     * We always add STREAM-START as the first token and STREAM-END as the last
     * token.
     */
    private def fetchStreamStart() {
        //We always add STREAM-START as the first token and STREAM-END as the
        //last token.

        //Read the token.
        val mark = getMark
        //Add STREAM-START.
        tokens += new StreamStartToken(mark, mark)
    }
    
    private def fetchStreamEnd() {
        // Set the current intendation to -1.
        unwindIndent(-1)

        // Reset simple keys.
        removePossibleSimpleKey()
        allowSimpleKey = false
        possibleSimpleKeys.clear

        // Read the token.
        val mark = getMark

        // Add STREAM-END.
        tokens += new StreamEndToken(mark, mark)

        // The stream is finished.
        done = true
    }
    
    private def fetchDirective() {
        // Set the current intendation to -1.
        unwindIndent(-1)

        // Reset simple keys.
        removePossibleSimpleKey()
        allowSimpleKey = false

        // Scan and add DIRECTIVE.
        tokens += scanDirective()
    }
    
    private def fetchDocumentStart() {
        fetchDocumentIndicator(true)
    }
    
    private def fetchDocumentEnd() {
        fetchDocumentIndicator(false)
    }
    
    private def fetchDocumentIndicator(isDocumentStart: Boolean) {
        // Set the current intendation to -1.
        unwindIndent(-1)

        // Reset simple keys. Note that there could not be a block collection
        // after '---'.
        removePossibleSimpleKey()
        allowSimpleKey = false

        // Add DOCUMENT-START or DOCUMENT-END.
        val startMark = getMark
        forward(3)
        val endMark = getMark
        tokens += (
            if (isDocumentStart) 
                new DocumentStartToken(startMark, endMark)
            else  new DocumentEndToken(startMark, endMark)
        )
    }
    
    private def fetchFlowSequenceStart() {
        fetchFlowCollectionStart(false)
    }
    
    private def fetchFlowMappingStart(){
        fetchFlowCollectionStart(true)
    }
    
    private def fetchFlowCollectionStart(isMappingStart: Boolean) {
        // '[' and '{' may start a simple key.
        savePossibleSimpleKey()

        // Increase the flow level.
        flowLevel += 1

        // Simple keys are allowed after '[' and '{'.
        allowSimpleKey = true

        // Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
        val startMark = getMark
        forward(1)
        val endMark = getMark
        tokens += (if (isMappingStart) {
            new FlowMappingStartToken(startMark, endMark)
        } else {
            new FlowSequenceStartToken(startMark, endMark)
        })
    }
    
    private def fetchFlowSequenceEnd() {
        fetchFlowCollectionEnd(false)
    }
    
    private def fetchFlowMappingEnd() {
        fetchFlowCollectionEnd(true)
    }
    
    private def fetchFlowCollectionEnd(isMappingEnd: Boolean) {
        // Reset possible simple key on the current level.
        removePossibleSimpleKey()

        // Decrease the flow level.
        this.flowLevel -= 1

        // No simple keys after ']' or '}'.
        this.allowSimpleKey = false

        // Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
        val startMark = getMark
        forward()
        val endMark = getMark
        this.tokens += (if (isMappingEnd) {
            new FlowMappingEndToken(startMark, endMark)
        } else {
            new FlowSequenceEndToken(startMark, endMark)
        })
    }
    
    private def fetchFlowEntry(){
        // Simple keys are allowed after ','.
        this.allowSimpleKey = true

        // Reset possible simple key on the current level.
        removePossibleSimpleKey()

        // Add FLOW-ENTRY.
        val startMark = getMark
        forward()
        val endMark = getMark
        this.tokens += new FlowEntryToken(startMark, endMark)
    }
    
    private def fetchBlockEntry() {
        // Block context needs additional checks.
        if (flowLevel == 0) {
            // Are we allowed to start a new entry?
            if (!allowSimpleKey) {
                throw new ScannerException(null, null, "sequence entries are not allowed here", getMark)
            }

            // We may need to add BLOCK-SEQUENCE-START.
            if (addIndent(column)) {
                val mark = getMark
                tokens += new BlockSequenceStartToken(mark, mark)
            }
        } else {
            // It's an error for the block entry to occur in the flow
            // context,but we let the parser detect this.
        }
        // Simple keys are allowed after '-'.
        allowSimpleKey = true

        // Reset possible simple key on the current level.
        removePossibleSimpleKey()

        // Add BLOCK-ENTRY.
        val startMark = getMark
        forward()
        val endMark = getMark
        tokens += new BlockEntryToken(startMark, endMark)
    }
    
    private def fetchKey() {
        // Block context needs additional checks.
        if (this.flowLevel == 0) {
            // Are we allowed to start a key (not necessary a simple)?
            if (!this.allowSimpleKey) {
                throw new ScannerException(null, null, "mapping keys are not allowed here", getMark)
            }
            // We may need to add BLOCK-MAPPING-START.
            if (addIndent(this.column)) {
                val mark = getMark
                this.tokens += new BlockMappingStartToken(mark, mark)
            }
        }
        // Simple keys are allowed after '?' in the block context.
        this.allowSimpleKey = this.flowLevel == 0

        // Reset possible simple key on the current level.
        removePossibleSimpleKey()

        // Add KEY.
        val startMark = getMark
        forward()
        val endMark = getMark
        this.tokens += new KeyToken(startMark, endMark)
    }
    
    private def fetchValue() {
        // Do we determine a simple key?
        if (this.possibleSimpleKeys.contains(this.flowLevel)) {
            // Add KEY.
            val key = this.possibleSimpleKeys(this.flowLevel)
            this.possibleSimpleKeys -= this.flowLevel
            this.tokens.insert(key.tokenNumber - this.tokensTaken, new KeyToken(key.mark, key.mark))

            // If this key starts a new block mapping, we need to add
            // BLOCK-MAPPING-START.
            if (this.flowLevel == 0) {
                if (addIndent(key.column)) {
                    this.tokens.insert(key.tokenNumber - this.tokensTaken,
                            new BlockMappingStartToken(key.mark, key.mark))
                }
            }
            // There cannot be two simple keys one after another.
            this.allowSimpleKey = false

        } else {// It must be a part of a complex key.
            // Block context needs additional checks.Do we really need them?
            // They
            // will be catched by the parser anyway.)
            if (this.flowLevel == 0) {

                // We are allowed to start a complex value if and only if we can
                // start a simple key.
                if (!this.allowSimpleKey) {
                    throw new ScannerException(null, null, "mapping values are not allowed here", getMark)
                }
            }

            // If this value starts a new block mapping, we need to add
            // BLOCK-MAPPING-START. It will be detected as an error later by
            // the parser.
            if (flowLevel == 0) {
                if (addIndent(column)) {
                    val mark = getMark
                    this.tokens += new BlockMappingStartToken(mark, mark)
                }
            }

            // Simple keys are allowed after ':' in the block context.
            allowSimpleKey = (flowLevel == 0)

            // Reset possible simple key on the current level.
            removePossibleSimpleKey()
        }
        // Add VALUE.
        val startMark = getMark
        forward()
        val endMark = getMark
        this.tokens += new ValueToken(startMark, endMark)
    }
    
    private def fetchAlias() {
        // ALIAS could be a simple key.
        savePossibleSimpleKey()

        // No simple keys after ALIAS.
        this.allowSimpleKey = false

        // Scan and add ALIAS.
        this.tokens += scanAnchor(false)
    }
    
    private def fetchAnchor() {
        // ANCHOR could start a simple key.
        savePossibleSimpleKey()

        // No simple keys after ANCHOR.
        this.allowSimpleKey = false

        // Scan and add ANCHOR.
        this.tokens += scanAnchor(true)
    }
    
    private def fetchTag() {
        // TAG could start a simple key.
        savePossibleSimpleKey()

        // No simple keys after TAG.
        this.allowSimpleKey = false

        // Scan and add TAG.
        this.tokens += scanTag()
    }
    
    private def fetchLiteral() {
        fetchBlockScalar('|')
    }
    
    private def fetchFolded() {
        fetchBlockScalar('>')
    }
    
    private def fetchBlockScalar(style: Char) {
        // A simple key may follow a block scalar.
        this.allowSimpleKey = true

        // Reset possible simple key on the current level.
        removePossibleSimpleKey()

        // Scan and add SCALAR.
        this.tokens += scanBlockScalar(style)
    }
    
    private def fetchSingle() {
        fetchFlowScalar('\'')
    }
    
    private def fetchDouble() {
        fetchFlowScalar('"')
    }
    
    private def fetchFlowScalar(style: Char) {
        // A flow scalar could be a simple key.
        savePossibleSimpleKey()

        // No simple keys after flow scalars.
        this.allowSimpleKey = false

        // Scan and add SCALAR.
        this.tokens += scanFlowScalar(style)
    }
    
    private def fetchPlain() {
        // A plain scalar could be a simple key.
        savePossibleSimpleKey()

        // No simple keys after plain scalars. But note that `scan_plain` will
        // change this flag if the scan is finished at the beginning of the
        // line.
        this.allowSimpleKey = false

        // Scan and add SCALAR. May change `allow_simple_key`.
        this.tokens += scanPlain()
    }
    
    // Checkers.
    
    private def checkDirective() = {
        // DIRECTIVE: ^ '%' ...
        // The '%' indicator is already checked.
        column == 0
    }
    
    private def checkDocumentStart(): Boolean = {
        // DOCUMENT-START: ^ '---' (' '|'\n')
        if (column == 0) {
            if ("---" == prefix(3) && Constant.NULL_BL_T_LINEBR.has(peek(3))) {
                return true
            }
        }
        false
    }
    
    private def checkDocumentEnd(): Boolean = {
        // DOCUMENT-END: ^ '...' (' '|'\n')
        if (column == 0) {
            if ("..." == prefix(3) && Constant.NULL_BL_T_LINEBR.has(peek(3))) {
                return true
            }
        }
        return false
    }
    
    private def checkBlockEntry() = {
        // BLOCK-ENTRY: '-' (' '|'\n')
        Constant.NULL_BL_T_LINEBR.has(peek(1))
    }
    
    private def checkKey() = {
        // KEY(flow context): '?'
        if (this.flowLevel != 0) {
            true
        } else {
            // KEY(block context): '?' (' '|'\n')
            Constant.NULL_BL_T_LINEBR.has(peek(1))
        }
    }
    
    private def checkValue() = {
        // VALUE(flow context): ':'
        if (flowLevel != 0) {
            true
        } else {
            // VALUE(block context): ':' (' '|'\n')
            Constant.NULL_BL_T_LINEBR.has(peek(1))
        }
    }
    
    private def checkPlain() = {
        /**
         * <pre>
         * A plain scalar may start with any non-space character except:
         *   '-', '?', ':', ',', '[', ']', '{', '}',
         *   '#', '&amp;', '*', '!', '|', '&gt;', '\'', '\&quot;',
         *   '%', '@', '`'.
         * 
         * It may also start with
         *   '-', '?', ':'
         * if it is followed by a non-space character.
         * 
         * Note that we limit the last rule to the block context (except the
         * '-' character) because we want the flow context to be space
         * independent.
         * </pre>
         */
        val ch = peek()
        Constant.NULL_BL_T_LINEBR.hasNo(ch, "-?:,[]{}#&*!|>\'\"%@`") || (Constant.NULL_BL_T_LINEBR.hasNo(peek(1)) 
                && (ch == '-' || (this.flowLevel == 0 && "?:".indexOf(ch) != -1)))
    }
    
    //Scanners
    
    /**
     * <pre>
     * We ignore spaces, line breaks and comments.
     * If we find a line break in the block context, we set the flag
     * `allow_simple_key` on.
     * The byte order mark is stripped if it's the first character in the
     * stream. We do not yet support BOM inside the stream as the
     * specification requires. Any such mark will be considered as a part
     * of the document.
     * TODO: We need to make tab handling rules more sane. A good rule is
     *   Tabs cannot precede tokens
     *   BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
     *   KEY(block), VALUE(block), BLOCK-ENTRY
     * So the checking code is
     *   if &lt;TAB&gt;:
     *       self.allow_simple_keys = False
     * We also need to add the check for `allow_simple_keys == True` to
     * `unwind_indent` before issuing BLOCK-END.
     * Scanners for block, flow, and plain scalars need to be modified.
     * </pre>
     */
    private def scanToNextToken() {
        if (index == 0 && peek() == '\uFEFF') {
            forward()
        }
        var found = false
        while (!found) {
            while (peek() == ' ') {
                forward()
            }
            if (peek() == '#') {
                while (Constant.NULL_OR_LINEBR.hasNo(peek())) {
                    forward()
                }
            }
            if (scanLineBreak().length() != 0) {
                if (this.flowLevel == 0) {
                    this.allowSimpleKey = true
                }
            } else {
                found = true
            }
        }
    }
    
    private def scanDirective() = {
        // See the specification for details.
        val startMark = getMark
        var endMark: Mark = null
        forward()
        var name = scanDirectiveName(startMark)
        var value: (_, _) = null
        if ("YAML" == name) {
            value = scanYamlDirectiveValue(startMark)
            endMark = getMark
        } else if ("TAG" == name) {
            value = scanTagDirectiveValue(startMark)
            endMark = getMark
        } else {
            endMark = getMark
            while (Constant.NULL_OR_LINEBR.hasNo(peek())) {
                forward()
            }
        }
        scanDirectiveIgnoredLine(startMark)
        new DirectiveToken(name, value, startMark, endMark)
    }
    
    private def scanDirectiveName(startMark: Mark) = {
        // See the specification for details.
        var length = 0
        var ch = peek(length)
        while (Constant.ALPHA.indexOf(ch) != -1) {
            length += 1
            ch = peek(length)
        }
        if (length == 0) {
            throw new ScannerException("while scanning a directive", startMark,
                    "expected alphabetic or numeric character, but found " + ch + "(" + ch.toInt
                            + ")", getMark)
        }
        var value = prefix(length)
        forward(length)
        ch = peek()
        if (Constant.NULL_BL_LINEBR.hasNo(ch)) {
            throw new ScannerException("while scanning a directive", startMark,
                    "expected alphabetic or numeric character, but found " + ch + "(" + ch.toInt
                            + ")", getMark)
        }
        value
    }
    
    private def scanYamlDirectiveValue(startMark: Mark) = {
        // See the specification for details.
        while (peek() == ' ') {
            forward()
        }
        val major = scanYamlDirectiveNumber(startMark)
        if (peek() != '.') {
            throw new ScannerException("while scanning a directive", startMark,
                    "expected a digit or '.', but found " + peek() + "("
                            + peek().toInt + ")", getMark)
        }
        forward()
        val minor = scanYamlDirectiveNumber(startMark)
        if (Constant.NULL_BL_LINEBR.hasNo(peek())) {
            throw new ScannerException("while scanning a directive", startMark,
                    "expected a digit or ' ', but found " + peek() + "("
                            + peek().toInt + ")", getMark)
        }
        (major, minor)
    }
    
    private def scanYamlDirectiveNumber(startMark: Mark): Int = {
        // See the specification for details.
        var ch = peek()
        if (!Character.isDigit(ch)) {
            throw new ScannerException("while scanning a directive", startMark,
                    "expected a digit, but found " + ch + "(" + ch.toInt + ")", getMark)
        }
        var length = 0
        while (Character.isDigit(peek(length))) {
            length += 1
        }
        var value = prefix(length).toInt
        forward(length)
        value
    }
    
    private def scanTagDirectiveValue(startMark: Mark) = {
        // See the specification for details.
        while (peek() == ' ') {
            forward()
        }
        val handle = scanTagDirectiveHandle(startMark)
        while (peek() == ' ') {
            forward()
        }
        val prefix = scanTagDirectivePrefix(startMark)
        (handle, prefix)
    }
    
    private def scanTagDirectiveHandle(startMark: Mark) = {
        // See the specification for details.
        val value = scanTagHandle("directive", startMark)
        val ch = peek()
        if (ch != ' ') {
            throw new ScannerException("while scanning a directive", startMark,
                    "expected ' ', but found " + peek() + "(" + ch + ")", getMark)
        }
        value
    }
    
    private def scanTagDirectivePrefix(startMark: Mark) = {
        // See the specification for details.
        val value = scanTagUri("directive", startMark)
        if (Constant.NULL_BL_LINEBR.hasNo(peek())) {
            throw new ScannerException("while scanning a directive", startMark,
                    "expected ' ', but found " + peek() + "(" + peek().toInt + ")",
                    getMark)
        }
        value
    }
    
    private def scanDirectiveIgnoredLine(startMark: Mark) = {
        // See the specification for details.
        while (peek() == ' ') {
            forward()
        }
        if (peek() == '#') {
            while (Constant.NULL_OR_LINEBR.hasNo(peek())) {
                forward()
            }
        }
        var ch = peek()
        if (Constant.NULL_OR_LINEBR.hasNo(ch)) {
            throw new ScannerException("while scanning a directive", startMark,
                    "expected a comment or a line break, but found " + ch + "(" + ch.toInt + ")", getMark)
        }
        scanLineBreak()
    }
    
    /**
     * <pre>
     * The specification does not restrict characters for anchors and
     * aliases. This may lead to problems, for instance, the document:
     *   [ *alias, value ]
     * can be interpreted in two ways, as
     *   [ &quot;value&quot; ]
     * and
     *   [ *alias , &quot;value&quot; ]
     * Therefore we restrict aliases to numbers and ASCII letters.
     * </pre>
     */
    private def scanAnchor(isAnchor: Boolean) = {
        val startMark = getMark
        val indicator = peek()
        val name = (if (indicator == '*') "alias" else "anchor")
        forward()
        var length = 0
        var ch = peek(length)
        while (Constant.ALPHA.contains(ch)) {
            length += 1
            ch = peek(length)
        }
        if (length == 0) {
            throw new ScannerException("while scanning an " + name, startMark,
                    "expected alphabetic or numeric character, but found but found " + ch, getMark)
        }
        val value = prefix(length)
        forward(length)
        ch = peek()
        if (Constant.NULL_BL_T_LINEBR.has(ch, "?:,]}%@`")) {
            throw new ScannerException("while scanning an " + name, startMark,
                    "expected alphabetic or numeric character, but found " + ch + "("
                            + ch.toInt + ")", getMark)
        }
        val endMark = getMark
        if (isAnchor) {
            new AnchorToken(value, startMark, endMark)
        } else {
            new AliasToken(value, startMark, endMark)
        }
    }
    
    private def scanTag() = {
        // See the specification for details.
        val startMark = getMark
        var ch = peek(1)
        var handle: String = null
        var suffix: String = null
        if (ch == '<') {
            forward(2)
            suffix = scanTagUri("tag", startMark)
            if (peek() != '>') {
                throw new ScannerException("while scanning a tag", startMark,
                        "expected '>', but found '" + peek() + "' (" + peek().toInt
                                + ")", getMark)
            }
            forward()
        } else if (Constant.NULL_BL_T_LINEBR.has(ch)) {
            suffix = "!"
            forward()
        } else {
            var length = 1
            var useHandle = false
            var process = true
            while (process && Constant.NULL_BL_LINEBR.hasNo(ch)) {
                if (ch == '!') {
                    useHandle = true
                    process = false
                }
                length += 1
                ch = peek(length)
            }
            handle = "!"
            if (useHandle) {
                handle = scanTagHandle("tag", startMark)
            } else {
                handle = "!"
                forward()
            }
            suffix = scanTagUri("tag", startMark)
        }
        ch = peek()
        if (Constant.NULL_BL_LINEBR.hasNo(ch)) {
            throw new ScannerException("while scanning a tag", startMark,
                    "expected ' ', but found '" + ch + "' (" + ch.toInt + ")", getMark)
        }
        val endMark = getMark
        new TagToken((handle, suffix), startMark, endMark)
    }
    
    private def scanBlockScalar(style: Char): Token = {
        // See the specification for details.
        var folded = false
        if (style == '>') {
            folded = true
        } else {
            folded = false
        }
        val chunks = new StringBuilder()
        val startMark = getMark
        // Scan the header.
        forward()
        val (chomping, increment) = scanBlockScalarIndicators(startMark)
        scanBlockScalarIgnoredLine(startMark)

        // Determine the indentation level and go to the first non-empty line.
        var minIndent = this.indent + 1
        if (minIndent < 1) {
            minIndent = 1
        }
        var breaks: String = null
        var maxIndent = 0
        var indent = 0
        var endMark: Mark = null
        if (increment == -1) {
            val (b, maxIndent, endMark) = scanBlockScalarIndentation()
            breaks = b
            indent = Math.max(minIndent, maxIndent)
        } else {
            indent = minIndent + increment - 1
            val (b, endMark) = scanBlockScalarBreaks(indent)
            breaks = b
        }

        var lineBreak = ""

        // Scan the inner part of the block scalar.
        var process = true
        while (process && this.column == indent && peek() != '\0') {
            chunks.append(breaks)
            var leadingNonSpace = " \t".indexOf(peek()) == -1
            var length = 0
            while (Constant.NULL_OR_LINEBR.hasNo(peek(length))) {
                length += 1
            }
            chunks.append(prefix(length))
            forward(length)
            lineBreak = scanLineBreak()
            val (b, endMark) = scanBlockScalarBreaks(indent)
            breaks = b
            if (this.column == indent && peek() != '\0') {

                // Unfortunately, folding rules are ambiguous.
                //
                // This is the folding according to the specification:
                if (folded && "\n" == lineBreak && leadingNonSpace
                        && " \t".indexOf(peek()) == -1) {
                    if (breaks.length() == 0) {
                        chunks.append(" ")
                    }
                } else {
                    chunks.append(lineBreak)
                }
                // Clark Evans's interpretation (also in the spec examples) not
                // imported from PyYAML
            } else {
                process = false
            }
        }
        // Chomp the tail.
        if (chomping.chompTailIsNotFalse) {
            chunks.append(lineBreak)
        }
        if (chomping.chompTailIsTrue) {
            chunks.append(breaks)
        }
        // We are done.
        new ScalarToken(chunks.toString(), false, startMark, endMark, style)
    }
    
    private def scanBlockScalarIndicators(startMark: Mark): (ChompTail, Int) = {
        // See the specification for details.
        var chomping: ChompTail = new ChompTail(true) //chomp not defined
        var increment = -1
        var ch = peek()
        if (ch == '-' || ch == '+') {
            if (ch == '+') {
                chomping = new ChompTail(false, true)
            } else {
                chomping =  new ChompTail(false, false)
            }
            forward()
            ch = peek()
            if (Character.isDigit(ch)) {
                increment = ch.toInt
                if (increment == 0) {
                    throw new ScannerException("while scanning a block scalar", startMark,
                            "expected indentation indicator in the range 1-9, but found 0", getMark)
                }
                forward()
            }
        } else if (Character.isDigit(ch)) {
            increment = ch.toInt
            if (increment == 0) {
                throw new ScannerException("while scanning a block scalar", startMark,
                        "expected indentation indicator in the range 1-9, but found 0", getMark)
            }
            forward()
            ch = peek()
            if (ch == '-' || ch == '+') {
                if (ch == '+') {
                    chomping = new ChompTail(false, true)
                } else {
                    chomping = new ChompTail(false, false)
                }
                forward()
            }
        }
        ch = peek()
        if (Constant.NULL_BL_LINEBR.hasNo(ch)) {
            throw new ScannerException("while scanning a block scalar", startMark,
                    "expected chomping or indentation indicators, but found " + ch, getMark)
        }
        (chomping, increment)
    }
    
    private def scanBlockScalarIgnoredLine(startMark: Mark): String = {
        // See the specification for details.
        while (peek() == ' ') {
            forward()
        }
        if (peek() == '#') {
            while (Constant.NULL_OR_LINEBR.hasNo(peek())) {
                forward()
            }
        }
        var ch = peek()
        if (Constant.NULL_OR_LINEBR.hasNo(ch)) {
            throw new ScannerException("while scanning a block scalar", startMark,
                    "expected a comment or a line break, but found " + ch, getMark)
        }
        scanLineBreak()
    }
    
    private def scanBlockScalarIndentation(): (String, Int, Mark) = {
        // See the specification for details.
        val chunks = new StringBuilder()
        var maxIndent = 0
        var endMark = getMark
        while (Constant.LINEBR.has(peek(), " \r")) {
            if (peek() != ' ') {
                chunks.append(scanLineBreak())
                endMark = getMark
            } else {
                forward()
                if (this.column > maxIndent) {
                    maxIndent = column
                }
            }
        }
        (chunks.toString(), maxIndent, endMark)
    }
    
    private def scanBlockScalarBreaks(indent: Int): (String, Mark) = {
        // See the specification for details.
        val chunks = new StringBuilder()
        var endMark = getMark
        while (this.column < indent && peek() == ' ') {
            forward()
        }
        while (Constant.FULL_LINEBR.has(peek())) {
            chunks.append(scanLineBreak())
            endMark = getMark
            while (this.column < indent && peek() == ' ') {
                forward()
            }
        }
        (chunks.toString(), endMark)
    }
    
    /**
     * <pre>
     * See the specification for details.
     * Note that we loose indentation rules for quoted scalars. Quoted
     * scalars don't need to adhere indentation because &quot; and ' clearly
     * mark the beginning and the end of them. Therefore we are less
     * restrictive then the specification requires. We only need to check
     * that document separators are not included in scalars.
     * </pre>
     */
    private def scanFlowScalar(style: Char): Token = {
        val _double = style == '"'
        val chunks = new StringBuilder()
        val startMark = getMark
        var quote = peek()
        forward()
        chunks.append(scanFlowScalarNonSpaces(_double, startMark))
        while (peek() != quote) {
            chunks.append(scanFlowScalarSpaces(startMark))
            chunks.append(scanFlowScalarNonSpaces(_double, startMark))
        }
        forward()
        val endMark = getMark
        new ScalarToken(chunks.toString(), false, startMark, endMark, style)
    }
    
    private def scanFlowScalarNonSpaces(_double: Boolean, startMark: Mark) = {
        // See the specification for details.
        val chunks = new StringBuilder()
        var process = true
        while (process) {
            var length = 0
            while (Constant.NULL_BL_T_LINEBR.hasNo(peek(length), "\'\"\\")) {
                length += 1
            }
            if (length != 0) {
                chunks.append(prefix(length))
                forward(length)
            }
            var ch = peek()
            if (!_double && ch == '\'' && peek(1) == '\'') {
                chunks.append("'")
                forward(2)
            } else if ((_double && ch == '\'') || (!_double && "\"\\".indexOf(ch) != -1)) {
                chunks.append(ch)
                forward()
            } else if (_double && ch == '\\') {
                forward()
                ch = peek()
                if (Constant.ESCAPE_REPLACEMENTS.contains(ch)) {
                    chunks.append(Constant.ESCAPE_REPLACEMENTS(ch))
                    forward()
                } else if (Constant.ESCAPE_CODES.contains(ch)) {
                    length = Constant.ESCAPE_CODES(ch)
                    forward()
                    val hex = prefix(length)
                    if (Constant.NOT_HEXA.findFirstIn(hex).isDefined) {
                        throw new ScannerException("while scanning a double-quoted scalar",
                                startMark, "expected escape sequence of " + length
                                        + " hexadecimal numbers, but found: " + hex, getMark)
                    }
                    val unicode: Char = Integer.parseInt(hex, 16).toChar
                    chunks.append(unicode)
                    forward(length)
                } else if (Constant.FULL_LINEBR.has(ch)) {
                    scanLineBreak()
                    chunks.append(scanFlowScalarBreaks(startMark))
                } else {
                    throw new ScannerException("while scanning a double-quoted scalar", startMark,
                            "found unknown escape character " + ch + "(" + ch.toInt + ")", getMark)
                }
            } else {
                process = false
            }
        }
        chunks.toString()
    }
    
    private def scanFlowScalarSpaces(startMark: Mark) = {
        // See the specification for details.
        val chunks = new StringBuilder()
        var length = 0
        while (" \t".indexOf(peek(length)) != -1) {
            length += 1
        }
        var whitespaces = prefix(length)
        forward(length)
        var ch = peek()
        if (ch == '\0') {
            throw new ScannerException("while scanning a quoted scalar", startMark,
                    "found unexpected end of stream", getMark)
        } else if (Constant.FULL_LINEBR.has(ch)) {
            var lineBreak = scanLineBreak()
            var breaks = scanFlowScalarBreaks(startMark)
            if ("\n" != lineBreak) {
                chunks.append(lineBreak)
            } else if (breaks.length() == 0) {
                chunks.append(" ")
            }
            chunks.append(breaks)
        } else {
            chunks.append(whitespaces)
        }
        chunks.toString()
    }
    
    private def scanFlowScalarBreaks(startMark: Mark) = {
        // See the specification for details.
        var chunks = new StringBuilder()
        var process = true
        while (process) {
            // Instead of checking indentation, we check for document
            // separators.
            var _prefix = prefix(3)
            if (("---" == _prefix || "..." == _prefix)
                    && Constant.NULL_BL_T_LINEBR.has(peek(3))) {
                throw new ScannerException("while scanning a quoted scalar", startMark,
                        "found unexpected document separator", getMark)
            }
            while (" \t".indexOf(peek()) != -1) {
                forward()
            }
            if (Constant.FULL_LINEBR.has(peek())) {
                chunks.append(scanLineBreak())
            } else {
                process = false
            }
        }
        chunks.toString()
    }
    
    /**
     * <pre>
     * See the specification for details.
     * We add an additional restriction for the flow context:
     *   plain scalars in the flow context cannot contain ',', ':' and '?'.
     * We also keep track of the `allow_simple_key` flag here.
     * Indentation rules are loosed for the flow context.
     * </pre>
     */
    private def scanPlain(): Token = {
        val chunks = new StringBuilder()
        val startMark = getMark
        var endMark = startMark
        var indent = this.indent + 1
        var spaces = ""
        var process1 = true
        while (process1) {
            var length = 0
            if (peek() == '#') {
                process1 = false
            } else {
                var ch = '\0'
                var process2 = true
                while (process2) {
                    var ch = peek(length)
                    if (Constant.NULL_BL_T_LINEBR.has(ch)
                            || (this.flowLevel == 0 && ch == ':' && Constant.NULL_BL_T_LINEBR
                                    .has(peek(length + 1)))
                            || (this.flowLevel != 0 && ",:?[]{}".indexOf(ch) != -1)) {
                        process2 = false
                    } else {
                    	length += 1
                    }
                }
                // It's not clear what we should do with ':' in the flow context.
                if (this.flowLevel != 0 && ch == ':'
                        && Constant.NULL_BL_T_LINEBR.hasNo(peek(length + 1), ",[]{}")) {
                    forward(length)
                    throw new ScannerException("while scanning a plain scalar", startMark,
                            "found unexpected ':'", getMark,
                            "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
                }
                if (length == 0) {
                    process1 = false
                } else {
                    this.allowSimpleKey = false
                    chunks.append(spaces)
                    chunks.append(prefix(length))
                    forward(length)
                    endMark = getMark
                    spaces = scanPlainSpaces()
                    if ("" == spaces || peek() == '#'
                            || (this.flowLevel == 0 && this.column < indent)) {
                        process1 = false
                    }
                }
            }
        }
        new ScalarToken(chunks.toString(), true, startMark, endMark)
    }
    
    /**
     * <pre>
     * See the specification for details.
     * The specification is really confusing about tabs in plain scalars.
     * We just forbid them completely. Do not use tabs in YAML!
     * </pre>
     */
    private def scanPlainSpaces(): String = {
        val chunks = new StringBuilder()
        var length = 0
        while (peek(length) == ' ') {
            length += 1
        }
        var whitespaces = prefix(length)
        forward(length)
        var ch = peek()
        if (Constant.FULL_LINEBR.has(ch)) {
            var lineBreak = scanLineBreak()
            this.allowSimpleKey = true
            var _prefix = prefix(3)
            if ("---" == _prefix || "..." == _prefix
                    && Constant.NULL_BL_T_LINEBR.has(peek(3))) {
                return ""
            }
            val breaks = new StringBuilder()
            while (Constant.LINEBR.has(peek(), " \r")) {
                if (peek() == ' ') {
                    forward()
                } else {
                    breaks.append(scanLineBreak())
                    _prefix = prefix(3)
                    if ("---" == _prefix || "..." == _prefix
                            && Constant.NULL_BL_T_LINEBR.has(peek(3))) {
                        return ""
                    }
                }
            }
            if ("\n" != lineBreak) {
                chunks.append(lineBreak)
            } else if (breaks == null || breaks.toString() == "") {
                chunks.append(" ")
            }
            chunks.append(breaks)
        } else {
            chunks.append(whitespaces)
        }
        chunks.toString()
    }
    
    /**
     * <pre>
     * See the specification for details.
     * For some strange reasons, the specification does not allow '_' in
     * tag handles. I have allowed it anyway.
     * </pre>
     */
    private def scanTagHandle(name: String, startMark: Mark): String = {
        var ch = peek()
        if (ch != '!') {
            throw new ScannerException("while scanning a " + name, startMark,
                    "expected '!', but found " + ch + "(" + ch.toInt + ")", getMark)
        }
        var length = 1
        ch = peek(length)
        if (ch != ' ') {
            while (Constant.ALPHA.contains(ch)) {
                length += 1
                ch = peek(length)
            }
            if (ch != '!') {
                forward(length)
                throw new ScannerException("while scanning a " + name, startMark,
                        "expected '!', but found " + ch + "(" + ch.toInt + ")", getMark)
            }
            length += 1
        }
        val value = prefix(length)
        forward(length)
        value
    }
    
    private def scanTagUri(name: String, startMark: Mark): String = {
        // See the specification for details.
        // Note: we do not check if URI is well-formed.
        val chunks = new StringBuilder()
        var length = 0
        var ch = peek(length)
        while (Constant.ALPHA.contains(ch) || "-;/?:@&=+$,_.!~*\'()[]%".contains(ch)) {
            if (ch == '%') {
                chunks.append(prefix(length))
                forward(length)
                length = 0
                chunks.append(scanUriEscapes(name, startMark))
            } else {
                length += 1
            }
            ch = peek(length)
        }
        if (length != 0) {
            chunks.append(prefix(length))
            forward(length)
            length = 0
        }
        if (chunks.length == 0) {
            throw new ScannerException("while scanning a " + name, startMark,
                    "expected URI, but found " + ch + "(" + ch.toInt + ")", getMark)
        }
        chunks.toString()
    }
    
    private def scanUriEscapes(name: String, startMark: Mark): String = {
        // See the specification for details.
        val bytes = new StringBuilder()
        while (peek() == '%') {
            forward()
            try {
                bytes.append(Integer.parseInt(prefix(2), 16))
            } catch {
                case nfe:NumberFormatException => throw new ScannerException("while scanning a " + name, startMark,
                        "expected URI escape sequence of 2 hexadecimal numbers, but found "
                                + peek(1) + "(" + peek(1).toInt + ") and "
                                + peek(2) + "(" + peek(2).toInt + ")", getMark)
            }
            forward(2)
        }
        bytes.toString()
    }
    
    private def scanLineBreak(): String = {
        // Transforms:
        // '\r\n' : '\n'
        // '\r' : '\n'
        // '\n' : '\n'
        // '\x85' : '\n'
        // default : ''
        var ch = peek()
        if ("\r\n\u0085".indexOf(ch) != -1) {
            if ("\r\n" == prefix(2)) {
                forward(2)
            } else {
                forward()
            }
            return "\n"
        } else if ("\u2028\u2029".indexOf(ch) != -1) {
            forward()
            return String.valueOf(ch)
        }
        return ""
    }
}


